(ns econews.core
  (:use [feedparser-clj.core]
        [econews.util])
  (:require
    [econews.url :as url]
    [econews.dict :as dict]
    [econews.path :as path]
    [clojure.core.matrix :as m]

    [feedparser-clj.core :as rss]
    [org.httpkit.client :as http]
    [clj-jb :as jb]
    [net.cgrand.enlive-html :as html]
    ;[taoensso.timbre :as tm]
            ))

(defn get-entries [url]
  (get (rss/parse-feed url) :entries))

(defn get-entry-title-sentence [entry]
  (jb/split (get entry :title)))

(defn sentence-weight [sentence word-weight]
  (average (map (fn [word] (get word-weight (get word :word) 0)) sentence))
  )

(defn unsignify [frequencies-count title-keywords]
  (into {} (map (fn [[word weight]]
                  [word (cond
                          (contains-val? dict/not-words word)
                          0
                          (re-find  #"\d+" word)
                          0
                          (re-find #"^\s*$" word)
                          0
                          (contains-val? dict/unsignificant-words word)
                          (/ weight 3)
                          (contains-val? title-keywords word)
                          (* weight 3)
                          :else
                          weight
                          )]) frequencies-count)))

(defn fetch-url [url]
  (html/html-resource (java.net.URL. url)))

(defn get-dom-path [dom path]
  (html/select dom path))


(defn sentence-split [sentence]
  (jb/split (clojure.string/trim sentence) :INDEX))

(defn get-nons [words]
  (map :word (filter (fn [word] (contains-val? ["n" "j" "nr" "nt" "nz" ""] (get word :type))) words)))

(defn article-keyword [words title-keywords]
  (take 7 (map first (sort-by second > (filter (fn [[_ v]] (> v 2)) (unsignify (frequencies (get-nons words))
                                                                               title-keywords)))))
  )

(defn get-sentence-rank [splitted-sentence keywords]
  (let [n-words (count splitted-sentence)
        n-keywords (count (filter (fn [word] (contains-val? keywords (get word :word)))
                                  splitted-sentence
                                  ))]
    (/ n-keywords n-words)
    )
  )

(defn generate-article-abstract [article title-keywords]
  (let [;keywords (article-keyword (sentence-split article))
        all-sentences (mapv clojure.string/trim (clojure.string/split article #"[。？！]"))
        splitted-sentences (jb/split-all all-sentences)
        keywords (article-keyword (apply concat splitted-sentences) title-keywords)
        sentence-rankings (mapv (fn [ss] (get-sentence-rank ss keywords)) splitted-sentences)
        top-five-bound (nth (sort > sentence-rankings) 4)
        abstract (clojure.string/join "\n" (for [i (range (count all-sentences))
                                                 :when (and (> (nth sentence-rankings i) top-five-bound)
                                                            (> (count (nth splitted-sentences i)) 5))
                                                 ]
                                             (nth all-sentences i)
                                             ))
        ]
    (str "关键词: " (clojure.string/join " " keywords) "\n"
         "摘要: \n" abstract
         )
    ))



(defn get-abstract-of-article [entry]
  (let [title (html/text entry)
        title-splitted (sentence-split title)
        title-keywords (get-nons title-splitted)
        href (get-in entry [:attrs :href])
        article (fetch-url href)
        date-time (get-dom-path article [:div.time])
        article-content (clojure.string/join (map html/text (get-dom-path article [:div#ContentBody.Body :p])))]
    (str title "\n"
         (first (get (first date-time) :content)) "\n"
         href "\n"
         (generate-article-abstract article-content title-keywords)
         ))
  )


(defn eastmoney-newspaper []
  (let [em-dom (fetch-url url/eastmoney-macro-news)
        em-marco-news-hl (get-dom-path em-dom path/eastmoney-macro-news-titles)]
    (clojure.string/join "\n---------------------------\n"
                         (map get-abstract-of-article em-marco-news-hl))
    )
  )

(defn get-newspaper []
  (let [
        ;sina-dom (fetch-url sina-finance)
        ;sina-headlines (get-dom-path sina-dom p-sina-headlines)
        ;sina-yaowen1 (get-dom-path sina-dom p-sina-yaowen1)
        ;sina-yaowen2 (get-dom-path sina-dom p-sina-yaowen2)
        ;sina-yaowen3 (get-dom-path sina-dom p-sina-yaowen3)
        ;
        ;caijing-dom (fetch-url caijing)
        ;caijing-yaowen (get-dom-path caijing-dom caijing-yaowen)
        ;
        ;caixin-dom (fetch-url caixin)
        ;caixin-economy (get-dom-path caixin-dom caixin-economy)
        ;
        ;ft-dom (fetch-url ftchinese)
        ;ft-headline-items (get-dom-path ft-dom ft-headline-items)
        ;
        ;wst-dom (fetch-url wallstreecn)
        ;wst-hot-news (get-dom-path wst-dom wst-hot-news)
        ]
    (println
      (eastmoney-newspaper)
      )
    )
  )


(defn get-news
  [url & urls]
  (let [;{:keys [title entries feed-type image language]} (rss/parse-feed url)
        entries (distinct-by :title (apply concat (map get-entries (cons url urls))))
        entry-titles (map :title entries)
        splitted-sentences (map jb/split entry-titles)
        all-words (apply concat splitted-sentences)
        all-nones (filter (fn [word] (= (get word :type) "n")) all-words)
        frequencies-count (frequencies (map :word all-nones))
        unsignified-weight (unsignify frequencies-count)
        words-ranking (sort-by val > unsignified-weight)
        news-ranking (map (fn [entry] [entry (sentence-weight (get-entry-title-sentence entry) unsignified-weight)]) entries)
        top-ten (keys (take 20 (sort-by val > (into {} news-ranking))))
        ]
    ;(println words-ranking)
    (println (clojure.string/join "\n" (map (fn [entry] (str (get entry :title) " " (get entry :published-date))) top-ten)))
    )
  )